package org.swu.swuse.Processor;

import org.swu.swuse.model.WebPage;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;

public class SwuProcessor implements PageProcessor {

	
	
	
	private Site site = Site.me().setRetryTimes(3).setSleepTime(100);

	public void process(Page page) {
		page.addTargetRequests(page.getHtml().links().all());
		
		
		WebPage webpage = new WebPage();
		webpage.setUrl(page.getUrl().toString());
		webpage.setHtml(page.getHtml().toString());
		webpage.setTitle(page.getHtml().$("title").toString());

		if (webpage.getUrl() == null) {
			page.setSkip(true);
		} else {
			page.putField("webpage", webpage);
		}
	}

	public Site getSite() {
		return site;
	}

}
